{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "import torch\n",
    "import numpy as np\n",
    "import sys\n",
    "sys.path.append('../../common/')\n",
    "\n",
    "from evaluator import *\n",
    "from utils import *\n",
    "from autoencoder import  *\n",
    "gpu_choice =get_default_device().index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_datase(dataset):\n",
    "    folder = os.path.join(\"../../processed\", dataset)\n",
    "    if not os.path.exists(folder):\n",
    "        raise Exception(\"Processed Data not found.\")\n",
    "    loader = []\n",
    "    for file in [\"train\", \"test\", \"labels\"]:\n",
    "        loader.append(np.load(os.path.join(folder, f\"{file}.npy\")))\n",
    "    ## 准备数据\n",
    "    train_data = loader[0]\n",
    "    test_data = loader[1]\n",
    "    labels = loader[2][:,0].reshape(-1,1)\n",
    "    return train_data, test_data, labels\n",
    "def convert_to_windows(data, window_size):\n",
    "    windows = []\n",
    "    length  = data.shape[0]\n",
    "    for i in range(length):\n",
    "        if length - i >= window_size:\n",
    "            window = data[i:i+window_size]\n",
    "        else:\n",
    "            window = data[i-window_size+1:i+1]\n",
    "        windows.append(window)    \n",
    "    return np.array(windows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SWaT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data, labels = load_datase(\"SWaT\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train data shape : (495000, 51)\n",
      "test data shape: (449919, 51)\n",
      "anomaly num : 54621\n",
      "anomaly ratio : 0.12140185233342002\n"
     ]
    }
   ],
   "source": [
    "print(f\"train data shape : {train_data.shape}\")\n",
    "print(f\"test data shape: {test_data.shape}\")\n",
    "print(f\"anomaly num : {np.count_nonzero(labels == 1)}\")\n",
    "ratio = np.count_nonzero(labels == 1)/labels.shape[0]\n",
    "print(f\"anomaly ratio : {ratio}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "window_size = 12\n",
    "batch_size = 1024\n",
    "num_epochs = 10\n",
    "z_dim = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 转换为时间窗口\n",
    "train_window = convert_to_windows(train_data, window_size)\n",
    "test_window = convert_to_windows(test_data, window_size)\n",
    "train_dataset = torch.from_numpy(train_window).float().view([train_window.shape[0], train_window.shape[1]*train_window.shape[2]])\n",
    "test_dataset =  torch.from_numpy(test_window).float().view([test_window.shape[0], test_window.shape[1]*test_window.shape[2]])\n",
    "train_gaussian_percentage  = 0.2\n",
    "indices = np.random.permutation(len(train_dataset))\n",
    "split_point = int(train_gaussian_percentage * len(train_dataset))\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, drop_last=True,\n",
    "                            sampler=SubsetRandomSampler(indices[:-split_point]), pin_memory=False)\n",
    "val_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, drop_last=True,\n",
    "                        sampler=SubsetRandomSampler(indices[-split_point:]),pin_memory=False)\n",
    "test_loader = DataLoader(dataset = test_dataset, batch_size=batch_size, shuffle=False, drop_last=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:49<00:00,  5.00s/it]\n"
     ]
    }
   ],
   "source": [
    "model = AutoEncoder(train_data[0].shape[0]*window_size, z_dim)\n",
    "history = model.fit(\n",
    "    train_loader=train_loader,\n",
    "    validation_loader=val_loader,\n",
    "    epochs=num_epochs\n",
    ")\n",
    "pred= model.predict_prob(test_loader)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_without_adjust = bf_search(labels.squeeze(), pred, verbose=False, is_adjust=False)\n",
    "results_adjust  = bf_search(labels.squeeze(), pred, verbose=False, is_adjust=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "without adjust\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'f1-score': 0.7418500857203862,\n",
       " 'precision': 0.9713430532920391,\n",
       " 'recall': 0.6000805549879935,\n",
       " 'TP': 32777.0,\n",
       " 'TN': 394331.0,\n",
       " 'FP': 967.0,\n",
       " 'FN': 21844.0,\n",
       " 'threshold': 0.44371801018714147}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"without adjust\")\n",
    "bf_search(labels.squeeze(), pred.squeeze(), verbose = False, is_adjust=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "point-adjust\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'f1-score': 0.8063285248800406,\n",
       " 'precision': 0.9980011342216576,\n",
       " 'recall': 0.6764248181694906,\n",
       " 'TP': 36947.0,\n",
       " 'TN': 395224.0,\n",
       " 'FP': 74.0,\n",
       " 'FN': 17674.0,\n",
       " 'threshold': 0.48644245749711434}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"point-adjust\")\n",
    "bf_search(labels.squeeze(), pred.squeeze(), verbose = False, is_adjust=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## WADI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data, labels = load_datase(\"WADI\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train data shape : (1209601, 123)\n",
      "test data shape: (172801, 123)\n",
      "anomaly num : 9860\n",
      "anomaly ratio : 0.0570598549776911\n"
     ]
    }
   ],
   "source": [
    "print(f\"train data shape : {train_data.shape}\")\n",
    "print(f\"test data shape: {test_data.shape}\")\n",
    "print(f\"anomaly num : {np.count_nonzero(labels == 1)}\")\n",
    "ratio = np.count_nonzero(labels == 1)/labels.shape[0]\n",
    "print(f\"anomaly ratio : {ratio}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "window_size = 50\n",
    "batch_size = 2048\n",
    "num_epochs = 10\n",
    "z_dim = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 转换为时间窗口\n",
    "train_window = convert_to_windows(train_data, window_size)\n",
    "test_window = convert_to_windows(test_data, window_size)\n",
    "train_dataset = torch.from_numpy(train_window).float().view([train_window.shape[0], train_window.shape[1]*train_window.shape[2]])\n",
    "test_dataset =  torch.from_numpy(test_window).float().view([test_window.shape[0], test_window.shape[1]*test_window.shape[2]])\n",
    "train_gaussian_percentage  = 0.2\n",
    "indices = np.random.permutation(len(train_dataset))\n",
    "split_point = int(train_gaussian_percentage * len(train_dataset))\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, drop_last=True,\n",
    "                            sampler=SubsetRandomSampler(indices[:-split_point]), pin_memory=False)\n",
    "val_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, drop_last=True,\n",
    "                        sampler=SubsetRandomSampler(indices[-split_point:]),pin_memory=False)\n",
    "test_loader = DataLoader(dataset = test_dataset, batch_size=batch_size, shuffle=False, drop_last=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [07:35<00:00, 45.54s/it]\n"
     ]
    }
   ],
   "source": [
    "model = AutoEncoder(train_data[0].shape[0]*window_size, z_dim)\n",
    "history = model.fit(\n",
    "    train_loader=train_loader,\n",
    "    validation_loader=val_loader,\n",
    "    epochs=num_epochs\n",
    ")\n",
    "pred= model.predict_prob(test_loader)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "without adjust\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'f1-score': 0.26717039942313714,\n",
       " 'precision': 0.8958333281491127,\n",
       " 'recall': 0.15699797144320693,\n",
       " 'TP': 1548.0,\n",
       " 'TN': 162761.0,\n",
       " 'FP': 180.0,\n",
       " 'FN': 8312.0,\n",
       " 'threshold': 585.0681762695308}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"without adjust\")\n",
    "bf_search(labels.squeeze(), pred.squeeze(), verbose = False, is_adjust=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "point-adjust\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'f1-score': 0.385528121547117,\n",
       " 'precision': 0.41003657932094584,\n",
       " 'recall': 0.36379310307931734,\n",
       " 'TP': 3587.0,\n",
       " 'TN': 157780.0,\n",
       " 'FP': 5161.0,\n",
       " 'FN': 6273.0,\n",
       " 'threshold': 584.6421508789062}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"point-adjust\")\n",
    "bf_search(labels.squeeze(), pred.squeeze(), verbose = False, is_adjust=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "yjq-3.6",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
